package com.share.example.spider.processor;

import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.share.example.spider.model.db.FundStock;
import com.share.example.spider.utils.WebClientUtil;
import lombok.extern.slf4j.Slf4j;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;

/**
 * @author 01401061 - 孙艳强
 * @date 2020/10/28 15:26
 */
@Slf4j
public class FundStockProcessor implements PageProcessor {


    /**
     * 1.解析数据
     * 2.获取下一页的链接
     * 3.通过page把数据推出去
     */
    @Override
    public void process(Page page) {
        String url = page.getUrl().get();

        List<FundStock> list = htmlUnit(url);

        page.putField(FundStock.class.getName(), list);
    }

    /*site的作用是限定爬取的网络位置*/
    @Override
    public Site getSite() {
        Site site = new Site();
        site.setDomain("fundf10.eastmoney.com");
        site.setUserAgent("User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36");
        return site;
    }


    private List<FundStock> htmlUnit(String url) {
        HtmlPage htmlPage = null;
        List<FundStock> list = new LinkedList<>();

        try {
            String name = this.getClass().getName() + "-" + Thread.currentThread().getName();
            htmlPage = WebClientUtil.getWebClient(name).getPage(url);//将客户端获取的树形结构转化为HtmlPage
            Thread.sleep(3000);//主线程休眠10秒 让客户端有时间执行js代码
        } catch (Exception e) {
            e.printStackTrace();
        }

        DomElement cctable = htmlPage.getElementById("cctable").getFirstElementChild();
        String h4 = cctable.getElementsByTagName("h4").get(0).getTextContent();
        String name = h4.split(" ")[0];
        HtmlElement htmlElement = cctable.getElementsByTagName("table").get(0).getElementsByTagName("tbody").get(0);
        DomNodeList<HtmlElement> tr = htmlElement.getElementsByTagName("tr");
        for (HtmlElement node : tr) {
            List<String> v = node.getElementsByTagName("td").stream().map(e -> e.getTextContent()).collect(Collectors.toList());

            FundStock fundStock = new FundStock();
            fundStock.setCreateTime(new Date());
            fundStock.setStockCode(v.get(1));
            fundStock.setStockName(v.get(2));

            int i = url.indexOf(".html");
            fundStock.setFundCode(url.substring(i-6,i));
            fundStock.setFundName(name);
            list.add(fundStock);
        }

        return list;
    }
}
